﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using AngleSharp;
using AngleSharp.Dom;

namespace UniversityPhoneCrawler
{
    class Program
    {
        // 存储提取的电话号码及其所属URL
        private static readonly Dictionary<string, HashSet<string>> PhoneNumbers = new Dictionary<string, HashSet<string>>();
        // 存储已访问的URL，避免重复爬取
        private static readonly HashSet<string> VisitedUrls = new HashSet<string>();
        // 控制并发数量的信号量
        private static readonly SemaphoreSlim Semaphore = new SemaphoreSlim(5); // 最多5个并发请求
        // 线程计数器
        private static int _processedUrlsCount = 0;
        private static int _validPhoneNumbersCount = 0;
        // 正则表达式模式匹配高校常见电话号码格式
        private static readonly Regex PhoneRegex = new Regex(
            @"((0\d{2,3})-)?(\d{7,8})|((0\d{2,3})\d{7,8})|(400-\d{3}-\d{4})|(400\d{7})",
            RegexOptions.Compiled | RegexOptions.IgnoreCase);

        static async Task Main(string[] args)
        {
            Console.WriteLine("=== 高校电话号码爬虫程序 ===");
            Console.Write("请输入搜索关键字: ");
            string keyword = Console.ReadLine();

            Console.WriteLine("开始搜索...");
            await SearchAndCrawl(keyword);

            Console.WriteLine("\n=== 爬取完成 ===");
            Console.WriteLine($"共处理 {_processedUrlsCount} 个URL");
            Console.WriteLine($"找到 {_validPhoneNumbersCount} 个有效电话号码");

            Console.WriteLine("\n显示部分爬取的URL:");
            int count = 0;
            foreach (var url in VisitedUrls.Take(20))
            {
                Console.WriteLine($"  {++count}. {url}");
            }

            Console.WriteLine("\n显示部分提取的电话号码:");
            count = 0;
            foreach (var phone in PhoneNumbers.Take(20))
            {
                Console.WriteLine($"  {++count}. {phone.Key} (来自 {phone.Value.Count} 个URL)");
            }

            Console.WriteLine("\n按任意键退出...");
            Console.ReadKey();
        }

        static async Task SearchAndCrawl(string keyword)
        {
            // 构建Bing搜索URL
            string searchUrl = $"https://cn.bing.com/search?q={Uri.EscapeDataString(keyword)} 电话";
            var config = Configuration.Default.WithDefaultLoader();
            var context = BrowsingContext.New(config);

            try
            {
                // 获取搜索结果页面
                var document = await context.OpenAsync(searchUrl);
                // 提取搜索结果中的URL
                var searchResults = document.QuerySelectorAll(".b_algo h2 a")
                    .Select(a => a.GetAttribute("href"))
                    .Where(url => !string.IsNullOrEmpty(url) && url.StartsWith("http"))
                    .ToList();

                Console.WriteLine($"找到 {searchResults.Count} 个搜索结果");

                var tasks = new List<Task>();
                foreach (var url in searchResults)
                {
                    tasks.Add(CrawlUrlAsync(url));
                }

                // 等待所有任务完成
                await Task.WhenAll(tasks);
            }
            catch (Exception ex)
            {
                Console.WriteLine($"搜索过程中出错: {ex.Message}");
            }
        }

        static async Task CrawlUrlAsync(string url)
        {
            // 获取信号量，控制并发数量
            await Semaphore.WaitAsync();

            try
            {
                // 检查URL是否已访问
                if (VisitedUrls.Contains(url))
                {
                    return;
                }

                // 添加到已访问列表
                lock (VisitedUrls)
                {
                    VisitedUrls.Add(url);
                }

                Interlocked.Increment(ref _processedUrlsCount);
                Console.WriteLine($"正在处理 ({_processedUrlsCount}): {url}");

                var config = Configuration.Default.WithDefaultLoader();
                var context = BrowsingContext.New(config);
                var document = await context.OpenAsync(url);

                // 提取页面中的所有文本
                string pageText = document.Body.TextContent;

                // 用正则表达式匹配电话号码
                var matches = PhoneRegex.Matches(pageText);

                // 筛选匹配结果
                lock (PhoneNumbers)
                {
                    foreach (Match match in matches)
                    {
                        string phoneNumber = match.Value;

                        // 过滤无效号码
                        if (IsValidPhoneNumber(phoneNumber))
                        {
                            if (!PhoneNumbers.TryGetValue(phoneNumber, out var urls))
                            {
                                urls = new HashSet<string>();
                                PhoneNumbers[phoneNumber] = urls;
                                Interlocked.Increment(ref _validPhoneNumbersCount);
                            }

                            urls.Add(url);
                        }
                    }
                }

                if (_validPhoneNumbersCount >= 100)
                {
                    Console.WriteLine("已找到100个有效电话号码，停止爬取...");
                    return;
                }

                // 提取页面中的链接进行深度爬取
                var links = document.QuerySelectorAll("a")
                    .Select(a => a.GetAttribute("href"))
                    .Where(href => !string.IsNullOrEmpty(href))
                    .ToList();

                // 处理相对URL
                var absoluteLinks = links.Select(link =>
                {
                    try
                    {
                        return new Uri(new Uri(url), link).ToString();
                    }
                    catch
                    {
                        return null;
                    }
                }).Where(link => link != null).ToList();

                // 对提取的链接进行广度优先爬取
                var tasks = new List<Task>();
                foreach (var link in absoluteLinks.Take(5)) // 每个页面只爬取前5个链接，避免无限爬取
                {
                    tasks.Add(CrawlUrlAsync(link));

                    // 控制爬取速度，避免被封IP
                    await Task.Delay(200);
                }

                await Task.WhenAll(tasks);
            }
            catch (Exception ex)
            {
                Console.WriteLine($"爬取 {url} 时出错: {ex.Message}");
            }
            finally
            {
                // 释放信号量
                Semaphore.Release();
            }
        }

        // 简单验证电话号码有效性
        static bool IsValidPhoneNumber(string phoneNumber)
        {
            // 排除常见的无效号码
            if (phoneNumber.Length < 7 || phoneNumber.Length > 15)
                return false;

            return true;
        }
    }
}

